from database.db_handler import MysqlHander
from common.my_http import MyHttp
import urllib.request
from bs4 import BeautifulSoup
from common.market_data import MarketData
import glob
import jieba
import re
import threading
import hashlib
import json
from common.entity_data import EntityData
from database.db_business import DbBusiness

'''
来源：https://www.xuanxiaodi.com/ranks/1921-1.html?hmsr=bd_rankcard
'''

class Daxuepaiming:
    def __init__(self):
        self.business = DbBusiness()
        self.entity = EntityData()
        self.entity_sets = ""
        self.no_rank = False
        self.data_len = 0
        self.rank_name = ""
        self.rank_year = 0
        self.rank_org = ""
    
    def get_url(self, rank_id, page_id):
        return "https://www.xuanxiaodi.com/api/school/ranking/list?id=" + str(rank_id) + "&regionId=&page=" + str(page_id) + "&pageSize=20"
    
    def get_online_data(self, url):
        proxy = self.business.query_proxy()
        json_data = json.loads(MyHttp.http_json_data_v2(url, proxy))
        if "err" in json_data:
            print(json_data["msg"])
            raise ValueError("error")

        self.rank_name = json_data["data"]["meta"]["title"]
        self.rank_year = json_data["data"]["meta"]["year"]
        self.rank_org = json_data["data"]["meta"]["org"]
        datas = json_data["data"]["data"]
        self.data_len = len(datas)
        for d in datas:
            self.entity_sets += "|" + d["chineseName"]
    
    def get_data(self):
        for rank_id in range (800, 2300):
            self.no_rank = False
            self.entity_sets = ""
            for page in range (1, 120):
                url = self.get_url(rank_id, page)
                try:
                    self.get_online_data(url)
                except Exception as e:
                    self.no_rank = True
                if self.data_len == 0:
                    break
            if self.no_rank:
                continue
            if self.entity_sets != "":
                self.entity.add_entity_data(self.rank_org + self.rank_name, "来源：" + str(self.rank_year) + self.rank_org + self.rank_name, 500, 5000001, "", "", self.entity_sets)
                self.entity.update_entity_data(self.rank_org + self.rank_name, "来源：" + str(self.rank_year) + self.rank_org + self.rank_name, 500, 5000001, "", "", self.entity_sets)


def daxuepaiming_api():
    d = Daxuepaiming()
    d.get_data()

if __name__ == '__main__':
    daxuepaiming_api()
